#!/bin/bash
#############################################################
# <next few lines under version control, D O  N O T  E D I T>
# $Date$
# $Author$
# $Revision$
# $Id$
#############################################################

#############################################################
# F U N C T I O N S   S T A R T   H E R E
#############################################################
clean_on_exit( ) {
	logfile=$1
	exp_name=$2
	\rm -f /tmp/msg
	echo "last 50 lines of ... ${logfile}" >> /tmp/msg
	tail -n 50 ${logfile}                  >> /tmp/msg
	mutt -s "${exp_name} has stopped on host `hostname -s`" `whoami`@atmos.uw.edu < /tmp/msg
	\rm -f /tmp/msg
	\rm -f sent
	exit
}

#############################################################
# M A I N   S C R I P T   S T A R T S   H E R E
#############################################################

param=${1}
until [ -e ${param} ]; do sleep 10; done
source ${param}

# define experiment source, run and log directories 
dir_src=${dir_expt_src}/${exp_name}/${run_type}/src
dir_run=${dir_expt_run}/${exp_name}/${run_type}/run${run_number}_${exp_type}
dir_run_control=${dir_expt_run}/${exp_name}/control/run${control_run_number}

# create run directories 
if [ -e ${dir_run} ]; then
	tmplog=/tmp/logfile
	\rm -f ${tmplog}; touch ${tmplog}
	echo "   ################################################"   >> ${tmplog}
	echo "   ${dir_run} exists"                                  >> ${tmplog}
	echo "   cannot continue, needs your attention, EXITING  "   >> ${tmplog}
	echo "   ################################################"   >> ${tmplog}
  echo "                                                   "   >> ${tmplog}
	clean_on_exit ${tmplog} ${exp_name}/${run_type}
else
	mkdir -p ${dir_run}; cd ${dir_run}
fi

logfile=${dir_run}/log_${run_type}_${run_number}
touch ${logfile}

echo "                                                             " >> ${logfile}
echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" >> ${logfile}
echo " ${run_type} for experiment ${exp_name}                      " >> ${logfile}
echo " started at `date`                                           " >> ${logfile}
echo " on host `hostname -s`                                       " >> ${logfile}
echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" >> ${logfile}
echo "                                                             " >> ${logfile}

# Copy model files in the run directory for the record
\cp -f ${dir_src}/fortran/Makefile     .
\cp -f ${dir_src}/fortran/SPECTRAL.f90 .
\cp -f ${dir_src}/fortran/fft.f90      .
\cp -f ${dir_src}/fortran/sqg_2b.f90   .
echo "   ############################################" >> ${logfile}
cat    ${dir_src}/fortran/SPECTRAL.f90                 >> ${logfile}
echo "   ############################################" >> ${logfile}
echo "                                               " >> ${logfile}

# Create a script to be submitted to the queue for each member
touch run_sQG_mem.scr; chmod 755 run_sQG_mem.scr
cat >> run_sQG_mem.scr << END_INPUT
#!/bin/bash

# Read input arguments
    mem=\${1}
dir_src=\${2}
dir_run=\${3}
dir_con=\${4}

# Create local run directory on host
dir_local=/home/\`hostname -s\`/\`whoami\`/sQG_mem_\${mem}
if [ -e \${dir_local} ]; then \rm -rf \${dir_local}; fi
mkdir -p \${dir_local}; cd \${dir_local}
logmem=\${dir_local}/log_sQG_mem\${mem}; touch \${logmem}
echo " "                                                  >> \${logmem}
echo "   ===============================================" >> \${logmem}
echo "      Processing member \${mem} on \`hostname\`"    >> \${logmem}
echo "   ===============================================" >> \${logmem}

# Copy model files
\cp -f \${dir_src}/fortran/Makefile     .
\cp -f \${dir_src}/fortran/SPECTRAL.f90 .
\cp -f \${dir_src}/fortran/fft.f90      .
\cp -f \${dir_src}/fortran/sqg_2b.f90   .

# Copy theta and tracer init files
\cp -f \${dir_src}/matlab/th_init.nc_${exp_type} th_init.nc
\cp -f \${dir_src}/matlab/tr_init.nc             .

# Copy ensemble member control file 
\cp -f \${dir_con}/smat_C_\${mem}.nc    .

# Copy base state file 
\cp -f \${dir_con}/base.nc              .

# Extract 1st time for restart.nc and remove base state from it
ncks   -h -F -d time,1,1,1 smat_C_\${mem}.nc -o tmp_restart.nc  >> \${logmem} 2>&1
ncdiff -h -v thetaB,thetaT tmp_restart.nc base.nc -o restart.nc >> \${logmem} 2>&1
if [ ! -e restart.nc ]; then
	echo "   ==============================================="  >> \${logmem}
	echo "      ncks or ncdiff FAILED to create restart.nc  "  >> \${logmem}
	echo "      for member \${mem} on \`hostname\`"            >> \${logmem}
	echo "   ==============================================="  >> \${logmem}
	\mv -f \${logmem} \${dir_run}/log_sQG_\${mem}
	cd; \rm -rf \${dir_local}
	exit
fi
\rm -f smat_C_\${mem}.nc base.nc

# Compile model and execute
make clean   >> \${logmem} 2>&1
make         >> \${logmem} 2>&1
(time ./sQG) >> \${logmem} 2>&1

if [ -e 'final+1.nc' ]; then
	echo "   ==============================================="     >> \${logmem}
	echo "      sQG SUCCESS for member \${mem} on \`hostname\`"   >> \${logmem}
	echo "   ==============================================="     >> \${logmem}
	\mv -f smat.nc \${dir_run}/smat_P_\${mem}.nc
else
	echo "   ==============================================="     >> \${logmem}
	echo "      sQG FAILED for member \${mem} on \`hostname\`"    >> \${logmem}
	echo "   ==============================================="     >> \${logmem}
fi
echo " "                                                        >> \${logmem}

# Move logfile and clean
\mv -f \${logmem} \${dir_run}/log_sQG_\${mem}
cd; \rm -rf \${dir_local}
END_INPUT

# Submit members to the queue
let ie=1
while [ ${ie} -le ${Ne} ]; do
	
	let emem='100000 + ie'
	emem=`echo ${emem} | cut -b2-`

	\cp -f run_sQG_mem.scr run_sQG_mem.scr_${emem}
	qsub -q MPI -pe smp 1 -V -o /dev/null -e /dev/null run_sQG_mem.scr_${emem} ${emem} ${dir_src} ${dir_run} ${dir_run_control} >> ${logfile} 2>&1
	\rm -f run_sQG_mem.scr_${emem}

	let ie='ie+1'
done
\rm -f run_sQG_mem.scr

# Wait for completion of all members
let ie=1
while [ ${ie} -le ${Ne} ]; do
	
	let emem='100000 + ie'
	emem=`echo ${emem} | cut -b2-`
	
	until [ -e ${dir_run}/log_sQG_${emem} ]; do sleep 10; done
	cat    ${dir_run}/log_sQG_${emem} >> ${logfile}
	\rm -f ${dir_run}/log_sQG_${emem}

	let ie='ie+1'
done

# Combine all smat_P_XXXXX.nc into smat_P_diag.nc
\cp -f ${dir_src}/fortran/Makefile     .
\cp -f ${dir_src}/fortran/sqg_diag.f90 .

make clean    >> ${logfile} 2>&1
make sqg_diag >> ${logfile} 2>&1

if [ ! -e sqg_diag ]; then
	echo "   sqg_diag did not get compiled correctly" >> ${logfile}
	clean_on_exit ${logfile} ${exp_name}/${run_type}
fi

\rm -f perturb.diag.*
cat >> perturb.diag.in << END_INPUT
${Ne}
'${dir_run}/'
'P'
END_INPUT

(time ./sqg_diag < perturb.diag.in) > perturb.diag.out 2>&1
cat perturb.diag.out >> ${logfile}

let success=`grep "sqg_diag done" perturb.diag.out | wc -l`
if [ $success -eq 0 ]; then
	echo "   sqg_diag failed to create smat_P_diag.nc" >> ${logfile}
	clean_on_exit ${logfile} ${exp_name}/${run_type}
else
	\rm -f perturb.diag.*
	while [ $ie -le $Ne ]; do
		let emem='100000 + ie'
		emem=`echo ${emem} | cut -b2-`
		\rm -f smat_P_${emem}.nc
		let ie='ie + 1'
	done
fi

# Calculate ensemble mean and spread of the difference
\cp -f ${dir_src}/fortran/Makefile      .
\cp -f ${dir_src}/fortran/sqg_stats.f90 .

make clean     >> ${logfile} 2>&1
make sqg_stats >> ${logfile} 2>&1

if [ ! -e sqg_stats ]; then
	echo "   sqg_stats did not get compiled correctly" >> ${logfile}
	clean_on_exit ${logfile} ${exp_name}/${run_type}
fi

\rm -f perturb.diag.*
cat >> perturb.diag.in << END_INPUT
'${dir_run_control}/'
'${dir_run}/'
END_INPUT

(time ./sqg_stats < perturb.diag.in) > perturb.diag.out 2>&1
cat perturb.diag.out >> ${logfile}

let success=`grep "sqg_stats done" perturb.diag.out | wc -l`
if [ $success -eq 0 ]; then
	echo "   sqg_stats failed to create smat_D_diag.nc" >> ${logfile}
	clean_on_exit ${logfile} ${exp_name}/${run_type}
else
	\rm -f perturb.diag.*
fi

echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" >> ${logfile}
echo " ${run_type} for experiment ${exp_name}"                       >> ${logfile}
echo " completed at `date`"                                          >> ${logfile}
echo " on host `hostname -s`"                                        >> ${logfile}
echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" >> ${logfile}

# Upon Exit, send email and clean
clean_on_exit ${logfile} ${exp_name}/${run_type}
